/*
 * Copyright (c) 2017 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* This is a private GC header which provides an implementation of      */
/* libatomic_ops subset primitives sufficient for GC assuming that GCC  */
/* atomic intrinsics are available (and have correct implementation).   */
/* This is enabled by defining GC_BUILTIN_ATOMIC macro.  Otherwise,     */
/* libatomic_ops library is used to define the primitives.              */

#ifndef GC_ATOMIC_OPS_H
#define GC_ATOMIC_OPS_H

#ifdef GC_BUILTIN_ATOMIC

#  include "gc/gc.h" /* for size_t */

#  ifdef __cplusplus
extern "C" {
#  endif

typedef size_t AO_t;

#  ifdef GC_PRIVATE_H /* have GC_INLINE */
#    define AO_INLINE GC_INLINE
#  else
#    define AO_INLINE static __inline
#  endif

#  if !defined(THREAD_SANITIZER) && !defined(GC_PRIVATE_H)
/* Similar to that in gcconfig.h.   */
#    if defined(__has_feature)
#      if __has_feature(thread_sanitizer)
#        define THREAD_SANITIZER
#      endif
#    elif defined(__SANITIZE_THREAD__)
#      define THREAD_SANITIZER
#    endif
#  endif /* !THREAD_SANITIZER && !GC_PRIVATE_H */

typedef unsigned char AO_TS_t;
#  define AO_TS_CLEAR 0
#  define AO_TS_INITIALIZER ((AO_TS_t)AO_TS_CLEAR)
#  if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
#    define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
#  else
#    define AO_TS_SET (AO_TS_t)1 /* true */
#  endif
#  define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
#  define AO_test_and_set_acquire(p) \
    (__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR)
#  define AO_HAVE_test_and_set_acquire

#  define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)

#  if defined(THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
/* Workaround a compiler warning (reported by gcc-11, at least)     */
/* that atomic_thread_fence is unsupported with thread sanitizer.   */
AO_INLINE void
AO_nop_full(void)
{
  volatile AO_TS_t dummy = AO_TS_INITIALIZER;
  (void)__atomic_test_and_set(&dummy, __ATOMIC_SEQ_CST);
}
#  else
#    define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
#  endif
#  define AO_HAVE_nop_full

#  define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_fetch_and_add
#  define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
#  define AO_HAVE_fetch_and_add1
#  define AO_fetch_and_sub1(p) AO_fetch_and_add(p, ~(AO_t)0 /* -1 */)
#  define AO_HAVE_fetch_and_sub1

#  define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_or

#  define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define AO_HAVE_load
#  define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
#  define AO_HAVE_load_acquire
/* AO_load_acquire_read(p) is not defined as it is unused, but we   */
/* need its AO_HAVE_x macro defined.                                */
#  define AO_HAVE_load_acquire_read

#  define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_store
#  define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
#  define AO_HAVE_store_release
#  define AO_store_release_write(p, v) AO_store_release(p, v)
#  define AO_HAVE_store_release_write

#  define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define AO_HAVE_char_load
#  define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define AO_HAVE_char_store
#  define AO_char_fetch_and_add1(p) __atomic_fetch_add(p, 1, __ATOMIC_RELAXED)
#  define AO_HAVE_char_fetch_and_add1

#  ifdef AO_REQUIRE_CAS
AO_INLINE int
AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
{
  return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELEASE,
                                          __ATOMIC_RELAXED /* on fail */);
}
#    define AO_HAVE_compare_and_swap_release
#  endif

#  ifdef __cplusplus
} /* extern "C" */
#  endif

#  ifndef NO_LOCKFREE_AO_OR
/* __atomic_or_fetch is assumed to be lock-free.    */
#    define HAVE_LOCKFREE_AO_OR 1
#  endif

#else
/* Fallback to libatomic_ops. */
#  include "atomic_ops.h"

/* AO_compiler_barrier, AO_load and AO_store should be defined for    */
/* all targets; the rest of the primitives are guaranteed to exist    */
/* only if AO_REQUIRE_CAS is defined (or if the corresponding         */
/* AO_HAVE_x macro is defined).  i686 and x86_64 targets have         */
/* AO_nop_full, AO_load_acquire, AO_store_release, at least.          */
#  if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
#    error AO_load or AO_store is missing; probably old version of atomic_ops
#  endif

#endif /* !GC_BUILTIN_ATOMIC */

#if defined(GC_BUILTIN_ATOMIC) || defined(__CHERI_PURE_CAPABILITY__)
/* Assume that GCC atomic intrinsics are available (and have correct  */
/* implementation).  p should be of a pointer to ptr_t (char*) value. */
#  define GC_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#  define GC_cptr_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
#  define GC_cptr_load_acquire_read(p) GC_cptr_load_acquire(p)
#  define GC_cptr_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
#  define GC_cptr_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
#  define GC_cptr_store_release_write(p, v) GC_cptr_store_release(p, v)
#  ifdef AO_REQUIRE_CAS
AO_INLINE int
GC_cptr_compare_and_swap(char *volatile *p, char *ov, char *nv)
{
  return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELAXED,
                                          __ATOMIC_RELAXED);
}
#  endif
#else
/* Redirect to the AO_ primitives.  Assume the size of AO_t matches   */
/* that of a pointer.                                                 */
#  define GC_cptr_load(p) (char *)AO_load((volatile AO_t *)(p))
#  define GC_cptr_load_acquire(p) (char *)AO_load_acquire((volatile AO_t *)(p))
#  define GC_cptr_load_acquire_read(p) \
    (char *)AO_load_acquire_read((volatile AO_t *)(p))
#  define GC_cptr_store(p, v) AO_store((volatile AO_t *)(p), (AO_t)(v))
#  define GC_cptr_store_release(p, v) \
    AO_store_release((volatile AO_t *)(p), (AO_t)(v))
#  define GC_cptr_store_release_write(p, v) \
    AO_store_release_write((volatile AO_t *)(p), (AO_t)(v))
#  ifdef AO_REQUIRE_CAS
#    define GC_cptr_compare_and_swap(p, ov, nv) \
      AO_compare_and_swap((volatile AO_t *)(p), (AO_t)(ov), (AO_t)(nv))
#  endif
#endif /* !GC_BUILTIN_ATOMIC */

#endif /* GC_ATOMIC_OPS_H */
